In [110]:
# VM Setup commands

def start_vm(kernel):
    !VM_CLI_IMAGE_PATH=/home/kpsingh/debian.img vm -q -k {kernel} && echo "Success"
    
def stop_vm():
    !pkill qemu

def vmrun(cmd):
    !ssh vm "{cmd}"
    
def pin_vm():
    !pin-vm -q

def pull(src, dest):
    !scp -r vm:{src} {dest}

def push(src, dest):
    !scp -r {src} vm:{dest}
In [106]:
import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
import numpy as np
import os 

import plotly
plotly.io.renderers.default='notebook'

NBINS = 50
FONT = dict(family="Courier New, monospace", size=18, color="White")
THEME = 'plotly_dark'

def format_name(n):
    n = n.replace("_", " ")
    return n.capitalize()


def savefig(fig, name):
    fig.write_image(
        os.path.join(PLOTS_DIR, name + '.jpg'), scale=4, height=400, width=1000
    )

def _generate_plot(data, unit):
    
    average = np.average(data)
    fig = px.histogram(data, marginal="violin", nbins=50)

    fig.update_layout(
        template=THEME,
        annotations=[
            go.layout.Annotation(
                x=average,
                y=1,
                font=FONT,
                yref="paper",
                text="Mean = %.2f %s" %(average, unit),
                showarrow=True,
                arrowhead=2,
                ax=0,
                ay=-40
            )
        ],
        shapes=[
            # Line Horizontal
            go.layout.Shape(
                type="line",
                yref="paper",
                x0=average,
                y0=0,
                x1=average,
                y1=1,
                line=dict(
                    color="palevioletred",
                    width=3,
                    #dash="dashdot",
                ),
            ),
        ],
        xaxis=go.layout.XAxis(
            title=go.layout.xaxis.Title(
                text="time",
                font=FONT,
            )
        ),
        yaxis=go.layout.YAxis(
            title=go.layout.yaxis.Title(
                text="Count",
                font=FONT,
            )
        )
    )
    # savefig(fig, name)
    return fig


def plot_hist(data, unit, range_x):
    fig = _generate_plot(data, unit)
    fig.update_xaxes(range=range_x)
    fig.show()
    
In [16]:
BASE_KERNEL="/home/kpsingh/static_call_analysis/base_bzImage"
SCALLS_KERNEL="/home/kpsingh/static_call_analysis/scalls_bzImage"
In [21]:
# eventfd tight loop benchmark
import os
import numpy as np
from scipy import stats


def run_eventfd(results, kernel):
    !cd ../workloads && make

    vm_out = os.path.join('/tmp', results)

    # Run and Pin the VM
    start_vm(kernel)
    vmrun("cat /proc/cmdline")
    vmrun("cat /sys/kernel/security/lsm")
    vmrun("/root/test_progs -t test_lsm")
    pin_vm()
    
    # Run the workload in the VM
    push("../workloads", "/tmp")
    vmrun("/tmp/workloads/run_eventfd.sh 100 " + vm_out)
    
    # Copy the output back, parse it and shut down the VM.
    output = os.path.join("./data", results)
    pull(vm_out, output)
    stop_vm()
    lines = open(output, 'r').read().strip().split('\n')
    return np.asarray(lines, dtype=float) / 1000

base_eventfd = run_eventfd("base_eventfd", BASE_KERNEL)
scalls_eventfd = run_eventfd("scalls_eventfd", SCALLS_KERNEL)
make: Nothing to be done for 'all'.
kernel_path=/home/kpsingh/static_call_analysis/base_bzImage image_path=/home/kpsingh/debian.img
Success
console=ttyS0,115200 root=/dev/sda rw nokaslr systemd.unit=multi-user.target isolcpus=2,3
capability,selinux,bpfCan't find bpf_testmod.ko kernel module: -2
WARNING! Selftests relying on bpf_testmod.ko will be skipped.
#212     test_lsm:OK
Summary: 1/0 PASSED, 0 SKIPPED, 0 FAILED
qemu_pid=370015
setting cpufreq governor to performance
Makefile                                      100%  693     2.6MB/s   00:00    
eventfd                                       100%   16KB  43.0MB/s   00:00    
run_eventfd.sh                                100%  261     1.8MB/s   00:00    
eventfd.c                                     100% 1095     8.2MB/s   00:00    
Running 100 iterations
base_eventfd                                  100%  600     2.5MB/s   00:00    
make: Nothing to be done for 'all'.
kernel_path=/home/kpsingh/static_call_analysis/scalls_bzImage image_path=/home/kpsingh/debian.img
Success
console=ttyS0,115200 root=/dev/sda rw nokaslr systemd.unit=multi-user.target isolcpus=2,3
capability,selinux,bpfCan't find bpf_testmod.ko kernel module: -2
WARNING! Selftests relying on bpf_testmod.ko will be skipped.
#212     test_lsm:OK
Summary: 1/0 PASSED, 0 SKIPPED, 0 FAILED
qemu_pid=370160
setting cpufreq governor to performance
Makefile                                      100%  693     1.4MB/s   00:00    
eventfd                                       100%   16KB  46.0MB/s   00:00    
run_eventfd.sh                                100%  261   949.6KB/s   00:00    
eventfd.c                                     100% 1095     7.5MB/s   00:00    
Running 100 iterations
scalls_eventfd                                100%  600     2.5MB/s   00:00    
In [107]:
stats_scall = stats.describe(scalls_eventfd)
stats_base = stats.describe(base_eventfd)

delta_pct = (stats_scall.mean - stats_base.mean) * 100 / stats_base.mean

print("Change in time to run the eventfd tightloop = {}".format(delta_pct))

range_x = stats.describe(np.union1d(base_eventfd, scalls_eventfd)).minmax

plot_hist(base_eventfd, "us", range_x)
plot_hist(scalls_eventfd, "us", range_x)
Change in time to run the eventfd tightloop = -9.70110521034551
In [79]:
# Unixbench
import os
from html.parser import HTMLParser

class MyHTMLParser(HTMLParser):

    def __init__(self):
        super().__init__()
        self.prev_table_id = 0
        self.curr_table_id = -1
        self.prev_row_id = 0
        self.curr_row_id = -1
        self.curr_cell_id = 0
        self.prev_cell_id = -1
        self.curr_tag = None
        self.curr_test = None
        self.results = {}
    
    def handle_starttag(self, tag, attrs):
        self.curr_tag = tag
        if tag == "table":
            self.curr_table_id = self.prev_table_id + 1
        if tag == "tr":
            self.curr_row_id = self.prev_row_id + 1
        if tag == "td":
            self.curr_cell_id = self.prev_cell_id + 1

    def handle_endtag(self, tag):
        self.curr_tag = None
        if tag == "table":
            self.prev_table_id = self.curr_table_id
            self.curr_table_id = -1
            self.prev_row_id = 0
            self.curr_row_id = -1
        if tag == "tr":
            self.prev_row_id = self.curr_row_id
            self.curr_row_id = -1
            self.prev_cell_id = 0
            self.curr_cell_id = -1
        if tag == "td":
            self.prev_cell_id = self.curr_cell_id
            self.curr_cell_id = -1

    def handle_data(self, data):
        if self.curr_tag == "td":
            return

        if self.curr_table_id == 2 and self.curr_row_id != 1:
            if self.curr_cell_id == 1:
                self.curr_test = data.strip()
            if self.curr_cell_id == 2:
                self.results[self.curr_test] = data.strip()
                self.curr_test = None
                
                
def _parse_unixbench(results):
    with open(results) as fh:
        parser = MyHTMLParser()
        parser.feed(fh.read())
        return parser.results

def run_unixbench(results, kernel):
    RESULT_FILE='result'
    
    cmd="cd /root/byte-unixbench-master/UnixBench && UB_RESULTDIR='/tmp/{}' UB_OUTPUT_FILE_NAME='{}' taskset -c 3 ./Run -i 4 -c 1 scalls".format(results, RESULT_FILE)
    vm_results = os.path.join('/tmp', results)

    # Run and Pin the VM
    start_vm(kernel)
    vmrun("uname -r")
    pin_vm()
    vmrun(cmd)
    pull(vm_results, './data')
    host_results = os.path.join('./data', results, RESULT_FILE + ".html")
    return _parse_unixbench(host_results)
    
stop_vm()    
scalls_unixbench = run_unixbench('scalls_unixbench', SCALLS_KERNEL)    
In [75]:
stop_vm()
base_unixbench = run_unixbench('base_unixbench', BASE_KERNEL)    
In [105]:
print("{:60} {}".format("Benchmark", "Delta: (+ is better)"))
for test in base_unixbench.keys():
    delta = (float(scalls_unixbench[test]) - float(base_unixbench[test])) * 100/ float(base_unixbench[test])
    print("{:60} {:+.4f}".format(test, delta))
Benchmark                                                    Delta: (+ is better)
Execl Throughput                                             +2.9015
File Write 1024 bufsize 2000 maxblocks                       +5.4196
Pipe Throughput                                              +7.7434
Pipe-based Context Switching                                 +3.5118
Process Creation                                             +0.3552
Shell Scripts (1 concurrent)                                 +1.7106
System Call Overhead                                         +3.0067
System Benchmarks Index Score (Partial Only):                +3.1809
In [115]:
# Dig into the "why?"

# eventfd tight loop benchmark
import os

def investigate(kernel, events):
    !cd ../workloads && make

    # Run and Pin the VM
    start_vm(kernel)
    vmrun("cat /proc/cmdline")
    vmrun("cat /sys/kernel/security/lsm")
    vmrun("/root/test_progs -t test_lsm")
    pin_vm()
    
    # Run the workload in the VM
    push("../workloads", "/tmp")
    vmrun("taskset -c 3 perf stat -e '{}' /tmp/workloads/eventfd".format(",".join(events)))
    stop_vm()
    
EVENTS = [
    'instructions',
    'branch-misses',
    'cache-misses',
    'branch-loads',
    'branch-load-misses',
]

investigate(BASE_KERNEL, EVENTS)
investigate(SCALLS_KERNEL, EVENTS)
make: Nothing to be done for 'all'.
kernel_path=/home/kpsingh/static_call_analysis/base_bzImage image_path=/home/kpsingh/debian.img
Success
console=ttyS0,115200 root=/dev/sda rw nokaslr systemd.unit=multi-user.target isolcpus=2,3
capability,selinux,bpfCan't find bpf_testmod.ko kernel module: -2
WARNING! Selftests relying on bpf_testmod.ko will be skipped.
#212     test_lsm:OK
Summary: 1/0 PASSED, 0 SKIPPED, 0 FAILED
qemu_pid=390098
setting cpufreq governor to performance
Makefile                                      100%  693     2.8MB/s   00:00    
eventfd                                       100%   16KB  47.3MB/s   00:00    
run_eventfd.sh                                100%  261     2.0MB/s   00:00    
eventfd.c                                     100% 1095     7.9MB/s   00:00    
17937

 Performance counter stats for '/tmp/workloads/eventfd':

          83317647      instructions                                                       
            607815      branch-misses                                                      
             29721      cache-misses                                                       
          21488975      branch-loads                                                       
            607833      branch-load-misses                                                 

       0.441201498 seconds time elapsed

       0.004320000 seconds user
       0.036722000 seconds sys


make: Nothing to be done for 'all'.
kernel_path=/home/kpsingh/static_call_analysis/scalls_bzImage image_path=/home/kpsingh/debian.img
Success
console=ttyS0,115200 root=/dev/sda rw nokaslr systemd.unit=multi-user.target isolcpus=2,3
capability,selinux,bpfCan't find bpf_testmod.ko kernel module: -2
WARNING! Selftests relying on bpf_testmod.ko will be skipped.
#212     test_lsm:OK
Summary: 1/0 PASSED, 0 SKIPPED, 0 FAILED
qemu_pid=390210
setting cpufreq governor to performance
Makefile                                      100%  693     2.7MB/s   00:00    
eventfd                                       100%   16KB  46.4MB/s   00:00    
run_eventfd.sh                                100%  261     1.8MB/s   00:00    
eventfd.c                                     100% 1095     7.4MB/s   00:00    
15590

 Performance counter stats for '/tmp/workloads/eventfd':

          80386675      instructions                                                       
            407264      branch-misses                                                      
             28298      cache-misses                                                       
          20281353      branch-loads                                                       
            407288      branch-load-misses                                                 

       0.431620687 seconds time elapsed

       0.005493000 seconds user
       0.025637000 seconds sys